{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.59368587,  0.        ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('MountainCarContinuous-v0', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "\n",
    "        reward = -1.0\n",
    "        if done:\n",
    "            reward = 100\n",
    "\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhp0lEQVR4nO3da1BTd/4/8PfJPVwS7olIqLRo8YoKCBG0VanUdW3Vzm630wed7k47trhT225nysy/dnZmZ+jsPtntbsc+2Nm2O3txx511t3UsLYJykSAEpCJarK0ULAQsmIRLCEnO5/+g5fyMWgW5nCR8XjNnxpzzTc4nB/Oec77nnO8RiIjAGGMyUMhdAGNs4eIAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJRrYAeuedd7BkyRLodDrk5+ejqalJrlIYYzKRJYD+9a9/4ZVXXsGbb76J1tZWZGdno6SkBAMDA3KUwxiTiSDHzaj5+fnIy8vDn/70JwCAKIqwWCz45S9/iddff32+y2GMyUQ13yucmJhAS0sLysrKpHkKhQLFxcWw2Wy3fY/X64XX65Vei6KIoaEhJCYmQhCEOa+ZMTY9RITh4WGkpqZCofjhA615D6Bvv/0WgUAAJpMpaL7JZMLnn39+2/eUl5fj17/+9XyUxxibRT09PUhLS/vB5fMeQPeirKwMr7zyivTa5XIhPT0dPT09MBgMMlbGGLsdt9sNi8WC2NjYO7ab9wBKSkqCUqlEf39/0Pz+/n6Yzebbvker1UKr1d4y32AwcAAxFsLu1kUy72fBNBoNcnJyUFVVJc0TRRFVVVWwWq3zXQ5jTEayHIK98soreOaZZ5Cbm4sNGzbg97//PUZHR/Hss8/KUQ5jTCayBNCTTz6Ja9eu4eDBg3A4HFi7di0qKipu6ZhmjEU2Wa4Dmim32w2j0QiXy8V9QIyFoKn+RsPiLBhjbO5NTHyDQOA61GoLlMooAKo5v86OA4gxBgBwOv+L3t7/B5UqBTpdFqKisqHXr4FOtxJqtQlKpRGT561mK5g4gBhj3xMRCDgRCDjh9V6Cy/UhABUUCh00Ggu02kxER2+AwVCC6Oi8WVkjBxBj7AcooFLFQaVaBL1+pbRHpFanztoaOIAYYwAAn08FnW4t9Prl34fNOmi1GVCrU6FQ6AAoZr1PiAOIMQYAaG1NxsaN/0NCwmIIgnJe1skjIjLGAAA+nwKCEDNv4QNwADHGZMQBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDd8Lxhi7qxsHTiWioJtSZ3KDKgcQYywIEcHtdmNkZAR9fX24du0axsfH4XK5oNfrMTIyAr1eD7VaDZ1Oh4SEBMTExGDx4sWIjY2FRqOZ8ro4gBhb4IgIPp8PQ0ND+OSTTyCKIjweD8xmM5KTk5GZmYn4+Hio1eqgxyyLoojx8XEMDQ2hr68PV65cgcvlQnR0NOLj46e0bg4gxhYgIoIoiujr68PZs2fx7bffQqfTIScnB+np6UEPAr3TIZbRaITJZEJWVpY0z+l0orOzc0p1cAAxtsB4vV5cunQJdrsdZrMZ69evR0pKClSqex+E/sb3xcfHY8WKFVN6HwcQYwvA5GFWe3s7zp8/j6ysLPzkJz9BTEyMrHVxADEW4QKBADo6OtDe3o7ly5fjqaeeglqtnvNH7kwFBxBjEYqI8M0336C5uRlJSUnYvXs3oqKiQiJ4JnEAMRaBJiYm0NTUhP7+fmzduhVxcXEhFTyTOIAYiyBEhN7eXpw8eRK5ubmwWq1QKudvjOfp4gBiLEIEAgG0tbWhv78fjz32GGJjY0Nyr+dG074XrLa2Frt27UJqaioEQcB///vfoOVEhIMHD2LRokXQ6/UoLi7GF198EdRmaGgITz/9NAwGA+Li4vCLX/wCIyMjM/oijC1ko6Oj+OSTTyAIAkpKSmAwGEI+fIB7CKDR0VFkZ2fjnXfeue3y3/72t3j77bfx7rvv4syZM4iOjkZJSQnGx8elNk8//TQ6OjpQWVmJY8eOoba2Fs8///y9fwvGFigiQk9PD06cOIF169Zh3bp1IX3IdQuaAQB09OhR6bUoimQ2m+l3v/udNM/pdJJWq6V//vOfRER04cIFAkDNzc1Sm48//pgEQaBvvvlmSut1uVwEgFwu10zKZyysiaJIn3/+Of373/+mkZEREkVR7pIkU/2NzupwHFeuXIHD4UBxcbE0z2g0Ij8/HzabDQBgs9kQFxeH3NxcqU1xcTEUCgXOnDlz28/1er1wu91BE2MLmd/vh81mw7Vr1/DYY48hOjo6LA65bjarAeRwOAAAJpMpaL7JZJKWORwOpKSkBC1XqVRISEiQ2tysvLwcRqNRmiwWy2yWzVhY8fl8qKurg1qtRmFhIdRqtdwl3bOwGJCsrKwMLpdLmnp6euQuibF5R0QYHx9HTU0N0tPTkZubG5Z7PTea1QAym80AgP7+/qD5/f390jKz2YyBgYGg5X6/H0NDQ1Kbm2m1WhgMhqCJsYWEiOD1elFZWYkHH3wQ999/f9iHDzDLAZSRkQGz2YyqqippntvtxpkzZ2C1WgEAVqsVTqcTLS0tUpvq6mqIooj8/PzZLIexiDE6Oorq6moUFBQgLS0tIsIHuIcLEUdGRnD58mXp9ZUrV9DW1oaEhASkp6fjwIED+M1vfoOlS5ciIyMDb7zxBlJTU7F7924AwPLly/Hoo4/iueeew7vvvgufz4f9+/fjZz/7GVJTU2ftizEWCYgIIyMjqKqqwubNm5GQkCB3SbNruqfXTp48SQBumZ555hki+u7U4BtvvEEmk4m0Wi1t27aNOjs7gz5jcHCQnnrqKYqJiSGDwUDPPvssDQ8PT7kGPg3PFgq3200ff/wxXb9+PaROs9/NVH+jAtENo02HCbfbDaPRCJfLxf1BLCIREZxOJ06fPo2ioiLExcXJXdK0TPU3GhZnwRhbSIgIY2NjOH36NKxWK4xGo9wlzRm+GZWxEDM+Po5Tp05h48aNkdfncxPeA2IshHi9XlRVVSE3N3fKT5YIZxxAjIWIySuc165di5SUlIg51X4nHECMhQCfz4eqqircf//9WLx48YIIH4ADiDHZERGamppgNpuRkZGxYMIH4ABiTFZEhPb2dmg0GmRnZy+o8AE4gBiTDRHh/Pnz6O3tRU5OzoILH4ADiDHZ9PX1obe3F1u3bg165vpCsjC/NWMym7whe/PmzdBoNHKXIxsOIMbmmcfjwfHjx2G1WqHT6eQuR1ZhfSV0GN7Gxha4QCCA2tpaPPTQQ0hKSpK7HNmF9R6Q2+3mEGJhQxRF2Gw2pKWl8dAz3wvrAGpoaMDExITcZTB2V0SEixcvYmJiAitWrFiQZ7xuJ6wDKCsrC1VVVfD7/XKXwtgPIiL09/ejp6cHmzZt4vC5QVgH0JIlS5CUlIS2tjY+FGMha2RkBDabDZs3bw7rJ1jMhbAOIEEQkJOTA7fbja+//ppDiIUcn8+HU6dOIS8vD3q9Xu5yQk5YBxAAKJVKFBUVwW634/r163KXw5hEFEU0NjZi2bJlC+oG0+kI+wACALVajYcffhh1dXXwer1yl8OYdJuFQqHA0qVLOXx+QEQEkCAISExMxOrVq9HQ0ABRFOUuiS1gRIRr167hypUryM/PX7C3WUxFxGwZQRCQkZGB6OhonD17lvuDmGw8Hg8aGxuxdetWqFRhfa3vnIuYAAK+C6H169fD4XDA4XBwCLF55/f7UV1djfXr1yM2NlbuckJeRAUQAKhUKmzZsgVnzpyBx+ORuxy2gBAR7HY7kpOTsXjxYrnLCQsRF0AAoNfrkZeXhxMnTsDn88ldDlsAiAhXr17F8PAwcnNzudN5iiIygARBQGpqKlJTU7k/iM0Lt9uNpqYmbNq0CUqlUu5ywkZEBhDwXQitW7cOw8PD6O7ulrscFsH8fj+qqqqQn58PrVYrdzlhJWIDCPjuIsXCwkLY7XYMDQ3xnhCbdZP9PsuWLUNaWhofek1TRAcQAGi1WlitVtTU1CAQCMhdDoswX375JcbGxrBixQq5SwlL0wqg8vJy5OXlITY2FikpKdi9ezc6OzuD2oyPj6O0tBSJiYmIiYnBE088gf7+/qA23d3d2LlzJ6KiopCSkoLXXnttzu5oFwQBixYtwtKlS2G323kviM0ap9OJpqYmFBQU8MWG92haW62mpgalpaVobGxEZWUlfD4ftm/fjtHRUanNyy+/jI8++ghHjhxBTU0Nent7sXfvXml5IBDAzp07MTExgYaGBnzwwQd4//33cfDgwdn7VjcRBAErV67EyMgIvvzySw4hNmM+nw/19fV45JFHEBUVJXc54YtmYGBggABQTU0NERE5nU5Sq9V05MgRqc3FixcJANlsNiIiOn78OCkUCnI4HFKbQ4cOkcFgIK/XO6X1ulwuAkAul2ta9Y6NjdHf//53crvd03ofYzcSRZFOnz5NHR0dJIqi3OWEpKn+Rme03+hyuQAACQkJAICWlhb4fD4UFxdLbbKyspCeng6bzQYAsNlsWL16NUwmk9SmpKQEbrcbHR0dt12P1+uF2+0Omu6FXq9HcXExamtreRAzdk+ICF988QUGBweRlZXFnc4zdM8BJIoiDhw4gMLCQqxatQoA4HA4oNFoEBcXF9TWZDLB4XBIbW4Mn8nlk8tup7y8HEajUZosFsu9lo3k5GRYLBbuD2L3ZHh4GBcuXMAjjzzC/T6z4J63YGlpKc6fP4/Dhw/PZj23VVZWBpfLJU09PT33/FmCIGDVqlW4du0aurq6OITYlPn9ftTW1qKwsHDBP05nttxTAO3fvx/Hjh3DyZMnkZaWJs03m82YmJiA0+kMat/f3w+z2Sy1ufms2OTryTY302q1MBgMQdNMKBQKFBcX49y5cxgZGZnRZ7GFgYhQW1uL1NRUfpzOLJpWABER9u/fj6NHj6K6uhoZGRlBy3NycqBWq1FVVSXN6+zsRHd3N6xWKwDAarWivb0dAwMDUpvKykoYDIZ5vZZCr9cjPz8f9fX13B/E7oiI0N3dDb/fj7Vr13K/z2yaTs/2Cy+8QEajkU6dOkV9fX3SNDY2JrXZt28fpaenU3V1NdntdrJarWS1WqXlfr+fVq1aRdu3b6e2tjaqqKig5ORkKisrm/Ue9rsRRZFaWlro9OnTfDaD/aDh4WH6z3/+Qx6PR+5SwsZUf6PTCiAAt53ee+89qY3H46EXX3yR4uPjKSoqivbs2UN9fX1Bn9PV1UU7duwgvV5PSUlJ9Oqrr5LP55v1LzcVfr+fPv30U+ru7p7xZ7HIMzExQUeOHKHe3l65SwkrU/2NCkTh1wvrdrthNBrhcrlm3B8EfDeCXUVFBbZv346oqCjexWYAvjv0amlpkQa64/8XUzfV3yifRwSg0+mwYcMGnDhxgseTZpKrV6/i+vXr3O8zhziA8H/jB6WlpeGzzz7jU/MMbrcbZ86cQWFhIY/vM4c4gL4nCALWrl2L/v5+9PX1yV0Ok1EgEEB9fT0KCgr4YYJzjAPoBkqlEps3b0Z9fX3QDbZs4SAitLW1YdGiRfwwwXnAAXSTqKgo6fog7g9aWIgIPT09uHbtGtasWcPhMw84gG4iCALS09ORmJiI9vZ27g9aQDweD86cOcPjOs8jDqDbmBxP+urVq/jmm284hBaAQCCA2tpabNy4EdHR0XKXs2BwAP0ApVKJoqIiNDU18fPFIhwR4bPPPkNKSgpSU1PlLmdB4QC6A6PRiLy8PNTX1/N40hGKiNDb24urV68iOzub+33mGQfQXaSlpSExMRGtra18KBaBPB4PmpubsW3bNh7fRwa8xe9CEARkZ2fD4XCgt7eXQyiC+P1+1NXVYcOGDYiOjua9HxlwAE2BSqXCtm3buD8ogtD3z/NKTEzEokWL5C5nweIAmqLJ8YNOnTrF4wdFgJ6eHoyMjGDdunW85yMjDqApmny+WHJyMj9vPowREYaHh9Ha2oqioiK+3kdmHEDTMDksg9vt5ufNh6lAIIDjx4+joKCAn+MeAjiApmnyefNtbW24fv067wmFEVEU0dTUhNWrV8NsNvOhVwjgALoHWq0WhYWFqKmp4f6gMEFEuHz5Mvx+P7KysuQuh32PA+geCIKAxMRErFy5EjabjW9aDXFEhIGBAdjtdn6Oe4jhv8Q9EgQBmZmZUCqV6Ojo4EOxEOb1etHY2Igf//jH0Gg0cpfDbsABNAOCIGDDhg24dOkSD2IWogKBAOrq6rB+/XrExsbKXQ67CQfQDKnVajz66KNobm7mhxyGmMlB5ePi4pCWlsadziGIA2gWREVFwWq1oqamBj6fT+5yGL4Ln6+++goej4efaBHCOIBmgSAISE5OxpIlS9DY2Mid0jIjIgwODuLChQvYuHEjX2wYwjiAZokgCFixYgW0Wi13SsvM4/Hgk08+QVFREVQqldzlsDvgAJpFgiAgNzcXPT096Orq4hCSgd/vR21tLbZu3Yr4+Hg+9ApxHECzTKFQ4KGHHsLZs2f5Sul5JooiGhoasGTJEpjNZrnLYVPAATQHoqOjUVxcjPr6eoyPj8tdzoJARGhvb4dSqcSDDz7Iez5hYloBdOjQIaxZswYGgwEGgwFWqxUff/yxtHx8fBylpaVITExETEwMnnjiCfT39wd9Rnd3N3bu3ImoqCikpKTgtddei8jbGWJjY5GTk4OKigpMTEzIXU5EIyJ0dXVhaGgIBQUFHD5hZFoBlJaWhrfeegstLS2w2+3YunUrHn/8cXR0dAAAXn75ZXz00Uc4cuQIampq0Nvbi71790rvDwQC2LlzJyYmJtDQ0IAPPvgA77//Pg4ePDi73yoETD7uefny5WhoaOAzY3OEiOBwONDR0cGPUQ5HNEPx8fH05z//mZxOJ6nVajpy5Ii07OLFiwSAbDYbEREdP36cFAoFORwOqc2hQ4fIYDCQ1+ud8jpdLhcBIJfLNdPy55woitTa2kp2u51EUZS7nIjjdrvpf//7H42MjMhdCrvBVH+j99wHFAgEcPjwYYyOjsJqtaKlpQU+nw/FxcVSm6ysLKSnp8NmswEAbDYbVq9eDZPJJLUpKSmB2+2W9qIizeSY0kNDQzh37hx3Ss+isbEx1NXVYdOmTfwsrzA17QBqb29HTEwMtFot9u3bh6NHj2LFihVwOBzQaDSIi4sLam8ymeBwOAAADocjKHwml08u+yFerxdutztoCicKhQJbtmzB0NAQn56fJV6vFxUVFcjOzr7l/xwLH9MOoAcffBBtbW04c+YMXnjhBTzzzDO4cOHCXNQmKS8vh9FolCaLxTKn65sLKpUKRUVFuHjxIhwOB4fQDAQCAdTX12P9+vVYvHgxdzqHsWkHkEajQWZmJnJyclBeXo7s7Gz84Q9/gNlsxsTEBJxOZ1D7/v5+6ZoMs9l8y1mxydd3um6jrKwMLpdLmnp6eqZbdkhQq9V4+OGHYbfbMTg4KHc5YWkyfNLS0nDffffJXQ6boRlfBySKIrxeL3JycqBWq1FVVSUt6+zsRHd3N6xWKwDAarWivb0dAwMDUpvKykoYDAasWLHiB9eh1WqlU/+TU7iKiorCli1b0NzcjOvXr8tdTlgRRRF2ux16vR7Lli3jPZ9IMJ2e7ddff51qamroypUrdO7cOXr99ddJEAT69NNPiYho3759lJ6eTtXV1WS328lqtZLVapXe7/f7adWqVbR9+3Zqa2ujiooKSk5OprKysjnpYQ9VoiiS0+mkw4cP09DQEJ8dm4JAIEB2u53a2tooEAjIXQ67i6n+RqcVQD//+c/pvvvuI41GQ8nJybRt2zYpfIiIPB4PvfjiixQfH09RUVG0Z88e6uvrC/qMrq4u2rFjB+n1ekpKSqJXX32VfD7fdMoI+wCaNDg4SBUVFeR2u+UuJaQFAgFqaGig06dPc/iEian+RgWi8OsNdbvdMBqNcLlcYX04BgDXr19HXV0dioqK+ObJ2xBFEe3t7ZiYmEBOTg6P5xwmpvob5b+mzOLi4mC1WtHQ0IDh4WG5ywkpoiiisbERPp+PwydC8V9UZpODmRUWFqKhoQFDQ0N8ih7fhU9raysAYP369Rw+EYr/qiEiLi4O+fn5OHHixII/RR8IBNDY2Ai1Ws2P0Ylw/JcNEYIgID4+Hjt27EBzczP6+voW5J6Qz+dDTU0NtFot1qxZw+ET4Xi8yhATExODLVu2oKqqCsuWLUNmZua0O6YDgQD+9re/wWg0Yvny5TCZTDAYDNLnhGpH9+joKOrr67F06VJkZGSEbJ1s9vBZsBDl8/lgs9lgNBqxevXqae0JXL16FYWFhejt7UV0dDTS09ORmZmJvLw8rFq1SgolvV4PpVIp+w+diOB2u1FZWYkNGzbAYrHIXhObman+RjmAQlggEEBTUxM8Hg82bdoEtVo9pffZ7XZYrdbbDvSmVqthMBhgsViwcuVKZGdnY926dbj//vthNpuh0+kgCMK8BQAR4euvv8a5c+dQVFSEhISEeVkvm1tT/Y3yIVgIUyqVKCgowJdffonKykoUFRUhNjb2ruHQ3NyMQCBw22U+nw+Dg4MYHBxEW1sb/vGPf0CpVCIhIQFpaWlYvXo1Hn30UTz55JNzHkKiKKKjowOXLl3Czp07odVq53R9LPRwAIU4QRDwwAMPwGAwoKKiAgUFBXc8RCEiXLhwYcod2EQEv9+PgYEBDAwMoLW1FS6XC08++eRsfo1b1un1elFfXw+DwYDHH3+cH5+zQPEphjAwea3QY489hs7OTtTX1//gONoej0e6fkatVsNoNCI6OnpaezM3Dio324gIAwMDOHbsGFJTU5GXl8fhs4DxXz5MCIIAnU6Hbdu2oaWlBR9++CE2b96MxMTEoHAZGxtDV1cXLBYLtm3bBpPJBK/Xi3PnzqGuru6uj45WqVRz1gk82bHudDpRUlKCmJgY7mxe4DiAwoxCoUBeXh4yMzNRXV2NlJQUFBQUQKVSQRAEnD9/HkSE3bt3IzExEQCg1+tRVFQEn8+H+vp6EBGSkpKwbNkyqNVqXLlyBVevXoUoioiPj0dOTs6s1kxE6OvrQ1NTE9LS0njweCbhAApTcXFxePzxx3Hu3Dl8+OGHyM3NhcViQVRUFHbt2nXL2aTJ4GptbUVycjL27NkjnZ3YuHEjqqur0dTUhJSUFMTGxs5anWNjY2hubkZvby927NgBo9HIez1MwgEUpgRBgEqlwrp165CZmYn6+nq0t7cjPz8fu3fvRlNT0y3v0Wq10tXWRqMxaP7DDz+Mq1evIi8vDzExMTOqbbJj+7PPPsOlS5eQm5uLoqIi3utht+AACnOCIMBgMGDHjh0YGBjAiRMncO3aNYiieMvFi8uXL8df//pX1NXV3XKWTKfTYfHixXjggQdmdPuD1+tFe3s7Ll++jAceeAB79+6FVqvlvR52WxxAEUIQBJhMJvz0pz9Fb28vmpqa0NnZKXU6m0wm7NixA0SExsbGW57WqlAosHHjRuzatWva6yYiDA8Po729HT09PUhLS8OuXbsQFRXFwcPuiAMowiiVSqSlpSE1NRXd3d3o6upCU1MTLBYLfD4fkpKSsHbtWjQ3NwftBaWnp+NXv/oV9Hr9Xdcx+T6Px4PLly/jwoULUCgUWLp0Kfbu3Qu1Ws3Bw6aEb8WIcJMX/XV3d6OtrQ0ejwcxMTH46quvMDY2BqVSiczMTDzyyCPSWbPbfYbf74fX68XQ0BC+/vprXL58GRqNBhaLBVlZWUhKSuI715mE7wVjtyAiTExMYGBgAF988QV6e3vR29uLpKQkqNVqxMbGQqfTwe/3Q6fTwel0QqFQYHR0FKIoQqfTITo6GkuXLkVycjLi4uI4dNht8b1g7BaCIECr1cJiscBisYCIIIoiRFHE8PAwPB4PfD4fvF4vNBoNiAgGgwExMTHSYdV83qjKIh8H0AImCAKUSqV0Mypj8433nxljsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJJixvxZi8f9btdstcCWPsdiZ/m3e71z0sA2hwcBAAYLFYZK6EMXYnw8PDQcP/3iwsA2jyxsnu7u47fjkWzO12w2KxoKenh4cxmSLeZvdmcpTM1NTUO7YLywCaHIPGaDTyf4p7YDAYeLtNE2+z6ZvKzgF3QjPGZMMBxBiTTVgGkFarxZtvvgmtVit3KWGFt9v08TabW2E5JjRjLDKE5R4QYywycAAxxmTDAcQYkw0HEGNMNmEZQO+88w6WLFkCnU6H/Px8NDU1yV2SbMrLy5GXl4fY2FikpKRg9+7d6OzsDGozPj6O0tJSJCYmIiYmBk888QT6+/uD2nR3d2Pnzp2IiopCSkoKXnvtNfj9/vn8KrJ56623IAgCDhw4IM3jbTZPKMwcPnyYNBoN/eUvf6GOjg567rnnKC4ujvr7++UuTRYlJSX03nvv0fnz56mtrY1+9KMfUXp6Oo2MjEht9u3bRxaLhaqqqshut1NBQQFt3LhRWu73+2nVqlVUXFxMZ8+epePHj1NSUhKVlZXJ8ZXmVVNTEy1ZsoTWrFlDL730kjSft9n8CLsA2rBhA5WWlkqvA4EApaamUnl5uYxVhY6BgQECQDU1NURE5HQ6Sa1W05EjR6Q2Fy9eJABks9mIiOj48eOkUCjI4XBIbQ4dOkQGg4G8Xu/8foF5NDw8TEuXLqXKykp66KGHpADibTZ/wuoQbGJiAi0tLSguLpbmKRQKFBcXw2azyVhZ6HC5XAD+74bdlpYW+Hy+oG2WlZWF9PR0aZvZbDasXr0aJpNJalNSUgK3242Ojo55rH5+lZaWYufOnUHbBuBtNp/C6mbUb7/9FoFAIOiPDgAmkwmff/65TFWFDlEUceDAARQWFmLVqlUAAIfDAY1Gg7i4uKC2JpMJDodDanO7bTq5LBIdPnwYra2taG5uvmUZb7P5E1YBxO6stLQU58+fR319vdylhLSenh689NJLqKyshE6nk7ucBS2sDsGSkpKgVCpvORvR398Ps9ksU1WhYf/+/Th27BhOnjyJtLQ0ab7ZbMbExAScTmdQ+xu3mdlsvu02nVwWaVpaWjAwMID169dDpVJBpVKhpqYGb7/9NlQqFUwmE2+zeRJWAaTRaJCTk4OqqippniiKqKqqgtVqlbEy+RAR9u/fj6NHj6K6uhoZGRlBy3NycqBWq4O2WWdnJ7q7u6VtZrVa0d7ejoGBAalNZWUlDAYDVqxYMT9fZB5t27YN7e3taGtrk6bc3Fw8/fTT0r95m80TuXvBp+vw4cOk1Wrp/fffpwsXLtDzzz9PcXFxQWcjFpIXXniBjEYjnTp1ivr6+qRpbGxMarNv3z5KT0+n6upqstvtZLVayWq1SssnTylv376d2traqKKigpKTkxfUKeUbz4IR8TabL2EXQEREf/zjHyk9PZ00Gg1t2LCBGhsb5S5JNgBuO7333ntSG4/HQy+++CLFx8dTVFQU7dmzh/r6+oI+p6uri3bs2EF6vZ6SkpLo1VdfJZ/PN8/fRj43BxBvs/nBw3EwxmQTVn1AjLHIwgHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2fx/gnyZfopVKoMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "env.action_space= Box(-1.0, 1.0, (1,), float32)\n",
      "state= [-0.46867633  0.        ]\n",
      "action= [-0.34565422]\n",
      "next_state= [-0.46960485 -0.00092854]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f65a13ce6a0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=2048,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-400.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(200_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/4.PPO.Mountain Car Continuous')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33.7, 1.0049875621120892)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/4.PPO.Mountain Car Continuous')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "414e1478",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhq0lEQVR4nO3da1BTd/4/8PcJ5MYlAbkkIqTSekEUb1wjtCpSqHW3rXZndrt90Gl32rHF3dp2OlNm/rWzMztDZ/dJt9uOfbBT2wc7a8cd7cVRWgQBkXBVFJHS0opgIUGFJFxCSHI+/wct+ZlqLSjm5PJ5zZyZ5pxvkk8OnnfP+Z5zvkcgIgJjjElAJnUBjLHwxQHEGJMMBxBjTDIcQIwxyXAAMcYkwwHEGJMMBxBjTDIcQIwxyXAAMcYkwwHEGJOMZAH0/vvvY+nSpVCpVMjPz0dra6tUpTDGJCJJAH3yySd49dVX8dZbb+HMmTNYt24dysrKMDIyIkU5jDGJCFLcjJqfn4/c3Fy89957AABRFJGWloY///nPeOONN/xdDmNMIpH+/sKZmRl0dHSgoqLCO08mk6GkpAQmk+mW73E6nXA6nd7XoihidHQUCQkJEAThntfMGJsfIsL4+DhSUlIgk/3ygZbfA+jatWvweDzQ6XQ+83U6Hb7++utbvqeyshJ//etf/VEeY2wBDQ4OIjU19ReX+z2A7kRFRQVeffVV72ubzQaDwYDBwUFoNBoJK2OM3YrdbkdaWhpiY2Nv287vAZSYmIiIiAhYLBaf+RaLBXq9/pbvUSqVUCqVN83XaDQcQIwFsF/rIvH7WTCFQoHs7GzU1NR454miiJqaGhiNRn+XwxiTkCSHYK+++iqeeeYZ5OTkIC8vD++88w4mJyfx7LPPSlEOY0wikgTQ73//e1y9ehX79u2D2WzG+vXrUVVVdVPHNGMstElyHdDdstvt0Gq1sNls3AfEWACa6zYaFGfBGGP33szMD/B4xiCXpyEiIgpA5D2/zo4DiDEGALBaP8XQ0P9DZGQyVKoMREWtg1q9FirVasjlOkREaDF73mqhgokDiDH2ExEejxUejxVO5zew2T4HEAmZTAWFIg1K5TJER+dBoylDdHTugnwjBxBj7BfIEBkZh8jIxVCrV3v3iOTylAX7Bg4gxhgAwOWKhEq1Hmr1qp/CZgOUynTI5SmQyVQAZAveJ8QBxBgDAJw5k4RNmz7DokVLIAgRfvlOHhGRMQYAcLlkEIQYv4UPwAHEGJMQH4IxFoZmrz+22WxoaWmBVquFFNckcwAxFkaICA6HAz09Pfj888/x2WefoaenB5s3b8Zzzz3n93o4gBgLcUQEURRx+fJlHD16FJ9++ik6Ojpgt9u9bdra2lBUVITS0lK/1sYBxFiIEkUR165dQ11dHQ4fPoz6+npYLJZbHmpZrVZ88skn2LNnj19r5ABiLIQQESYmJnD27FkcPnwYVVVV+P777+FyuX71vWNjY37vB+IAYiwEEBH6+vpw+PBhfPLJJ+jp6cH09PRt3yOTyRAdHQ0AmJqagtVqxfnz57Flyxa/PeyBA4ixEPH222/jwIEDc9qLiYmJQXFxMZYvXw6ZTIb+/n5UV1fjww8/xJYtW+59sT/h64AYCxGPPvrobR+BMysiIgKlpaXYsGEDYmNjER0djczMTOTl5eH06dMYHh72Q7U/4gBiLAQIgoDi4mJkZmb6zNPpdFizZg3S09OhUCgAAIsXL0ZmZqbPYZYgCNBoNPjLX/4CrVbrt7r5EIyxEBEXF4cdO3agq6sLgiDAaDSiqKgIarUabrcb3333HT777DNERkYiIuLm2y1kMhk2bdrk7Re60Y2HdUR0U3jdKQ4gxkKEIAjYtWsX3nnnHaSlpWHLli3evR65XI6VK1di27Zt6O/vv2VoqNVq6HQ6EBHsdjsmJiYwPDyMq1evYnp6GjabDWq1GhMTE1Cr1ZDL5VCpVFi0aBFiYmKwZMkSxMbGer9zLjiAGAshWVlZKCgogEajgVwu91kmCAI2bdqE9957D06nE0ePHsX4+DgAIDIyEvHx8WhqakJjYyMcDgf0ej2SkpKwbNkyxMfHQy6X+/QxiaKI6elpjI6OYnh4GJcuXYLNZkN0dDTi4+PnVC8HEGMhRKlU4ne/+x2++uqrWy5Xq9VITk6GSqXCH//4RzQ2NuLcuXOIiYnBxo0bkZ2dDYPB4PMg0NsdYmm1Wuh0OmRkZHjnWa1W9Pb2zqleDiDGQoggCNi5cyfGx8chCILPtUCCIGD9+vUAgAsXLqC9vR16vR67d+9GcnIyIiPvfBD6G98XHx/v0xl+2/fxY3kYCz1EhK6uLtTV1cFms0GpVGLFihXQ6XT45ptvkJGRgdWrVyMmJuaefD8/loexMCYIArKyspCeno6xsTH09fXhhx9+gMFgwFNPPQW5XO63q51vhwOIsRBms9nQ0dGBxMRE7Nq1C1FRUQERPLM4gBgLQTMzM2htbYXFYkFxcTHi4uICKnhmcQAxFkKICENDQzh58iRycnJgNBpvedFhoOAAYixEeDwedHZ2wmKx4LHHHkNsbGxA7vXcaN73gjU0NOC3v/0tUlJSIAgCPv30U5/lRIR9+/Zh8eLFUKvVKCkpwbfffuvTZnR0FE8//TQ0Gg3i4uLwpz/9CRMTE3f1QxgLZ5OTk/jyyy8hCALKysqg0WgCPnyAOwigyclJrFu3Du+///4tl//973/Hu+++iw8++AAtLS2Ijo5GWVmZz/UITz/9NLq7u1FdXY2jR4+ioaEBL7zwwp3/CsbCFBFhcHAQJ06cwIYNG7Bhw4aAPuS6Cd0FAHTkyBHva1EUSa/X0z/+8Q/vPKvVSkqlkv773/8SEdHFixcJALW1tXnbHD9+nARBoB9++GFO32uz2QgA2Wy2uymfsaAmiiJ9/fXX9L///Y8mJiZIFEWpS/Ka6za6oMNxXLp0CWazGSUlJd55Wq0W+fn5MJlMAACTyYS4uDjk5OR425SUlEAmk6GlpeWWn+t0OmG3230mxsKZ2+2GyWTC1atX8dhjjyE6OjooDrl+bkEDyGw2AwB0Op3PfJ1O511mNpuRnJzsszwyMhKLFi3ytvm5yspKaLVa75SWlraQZTMWVFwuF06dOgW5XI7CwsKbbjoNJkExIFlFRQVsNpt3GhwclLokxvyOiDA9PY36+noYDAbk5OQE5V7PjRY0gPR6PQDAYrH4zLdYLN5ler0eIyMjPsvdbjdGR0e9bX5OqVRCo9H4TIyFEyKC0+lEdXU1Vq5cifvvvz/owwdY4ABKT0+HXq9HTU2Nd57dbkdLSwuMRiMAwGg0wmq1oqOjw9umtrYWoigiPz9/IcthLGRMTk6itrYWBQUFSE1NDYnwAe7gQsSJiQn09fV5X1+6dAmdnZ1YtGgRDAYD9u7di7/97W9Yvnw50tPT8eabbyIlJQVPPPEEAGDVqlV45JFH8Pzzz+ODDz6Ay+XCnj178Ic//AEpKSkL9sMYCwX003O+ampq8NBDD2HRokVSl7Sw5nt67eTJkwTgpumZZ54hoh9PDb755puk0+lIqVTStm3bqLe31+czrl+/Tk899RTFxMSQRqOhZ599lsbHx+dcA5+GZ+HCbrfT8ePHaWxsLKBOs/+auW6jPB4QYwGIiGC1WnH69GkUFRUhLi5O6pLmZa7baFCcBWMsnBARpqamcPr0aRiNRr8+Jsff+GZUxgLM9PQ06urqsGnTptDr8/kZ3gNiLIA4nU7U1NQgJydnzk+WCGYcQIwFiNkrnNevX4/k5OSQOdV+OxxAjAUAl8uFmpoa3H///ViyZElYhA/AAcSY5IgIra2t0Ov1SE9PD5vwATiAGJMU/fT4HIVCgXXr1oVV+AAcQIxJhohw4cIFDA0NITs7O+zCB+AAYkwyw8PDGBoaQnFxsc8z18NJeP5qxiQ2e0P2Qw89BIVCIXU5kuEAYszPHA4Hjh07BqPRCJVKJXU5kgrqK6GD8DY2FuY8Hg8aGhqwefNmJCYmSl2O5IJ6D8hut3MIsaAhiiJMJhNSU1N56JmfBHUANTU1YWZmRuoyGPtVRISenh7MzMwgMzMzLM943UpQB1BGRgZqamrgdrulLoWxX0REsFgsGBwcxIMPPsjhc4OgDqClS5ciMTERnZ2dfCjGAtbExARMJhMeeuihoH6Cxb0Q1AEkCAKys7Nht9tx+fJlDiEWcFwuF+rq6pCbmwu1Wi11OQEnqAMIACIiIlBUVIT29naMjY1JXQ5jXqIoorm5GStWrAirG0znI+gDCADkcjm2bNmCU6dOwel0Sl0OY97bLGQyGZYvX87h8wtCIoAEQUBCQgKysrLQ1NQEURSlLomFMSLC1atXcenSJeTn54ftbRZzETJrRhAEpKenIzo6GmfPnuX+ICYZh8OB5uZmFBcXIzIyqK/1vedCJoCAH0No48aNMJvNMJvNHELM79xuN2pra7Fx40bExsZKXU7AC6kAAoDIyEhs3boVLS0tcDgcUpfDwggRob29HUlJSViyZInU5QSFkAsgAFCr1cjNzcWJEyfgcrmkLoeFASLClStXMD4+jpycHO50nqOQDCBBEJCSkoKUlBTuD2J+Ybfb0draigcffBARERFSlxM0QjKAgB9DaMOGDRgfH8fAwIDU5bAQ5na7UVNTg/z8fCiVSqnLCSohG0DAjxcpFhYWor29HaOjo7wnxBbcbL/PihUrkJqayode8xTSAQQASqUSRqMR9fX18Hg8UpfDQsx3332HqakpZGZmSl1KUJpXAFVWViI3NxexsbFITk7GE088gd7eXp8209PTKC8vR0JCAmJiYvDkk0/CYrH4tBkYGMCOHTsQFRWF5ORkvP766/fsjnZBELB48WIsX74c7e3tvBfEFozVakVraysKCgr4YsM7NK+1Vl9fj/LycjQ3N6O6uhoulwulpaWYnJz0tnnllVfwxRdf4NChQ6ivr8fQ0BB27drlXe7xeLBjxw7MzMygqakJH3/8MT766CPs27dv4X7VzwiCgNWrV2NiYgLfffcdhxC7ay6XC42NjXj44YcRFRUldTnBi+7CyMgIAaD6+noiIrJarSSXy+nQoUPeNj09PQSATCYTEREdO3aMZDIZmc1mb5v9+/eTRqMhp9M5p++12WwEgGw227zqnZqaov/85z9kt9vn9T7GbiSKIp0+fZq6u7tJFEWpywlIc91G72q/0WazAQAWLVoEAOjo6IDL5UJJSYm3TUZGBgwGA0wmEwDAZDIhKysLOp3O26asrAx2ux3d3d23/B6n0wm73e4z3Qm1Wo2SkhI0NDTwIGbsjhARvv32W1y/fh0ZGRnc6XyX7jiARFHE3r17UVhYiDVr1gAAzGYzFAoF4uLifNrqdDqYzWZvmxvDZ3b57LJbqayshFar9U5paWl3WjaSkpKQlpbG/UHsjoyPj+PixYt4+OGHud9nAdzxGiwvL8eFCxdw8ODBhaznlioqKmCz2bzT4ODgHX+WIAhYs2YNrl69iv7+fg4hNmdutxsNDQ0oLCwM+8fpLJQ7CqA9e/bg6NGjOHnyJFJTU73z9Xo9ZmZmYLVafdpbLBbo9Xpvm5+fFZt9Pdvm55RKJTQajc90N2QyGUpKSnD+/HlMTEzc1Wex8EBEaGhoQEpKCj9OZwHNK4CICHv27MGRI0dQW1uL9PR0n+XZ2dmQy+Woqanxzuvt7cXAwACMRiMAwGg0oqurCyMjI9421dXV0Gg0fr2WQq1WIz8/H42NjdwfxG6LiDAwMAC3243169dzv89Cmk/P9osvvkharZbq6upoeHjYO01NTXnb7N69mwwGA9XW1lJ7ezsZjUYyGo3e5W63m9asWUOlpaXU2dlJVVVVlJSURBUVFQvew/5rRFGkjo4OOn36NJ/NYL9ofHycDh8+TA6HQ+pSgsZct9F5BRCAW04HDhzwtnE4HPTSSy9RfHw8RUVF0c6dO2l4eNjnc/r7+2n79u2kVqspMTGRXnvtNXK5XAv+4+bC7XbTV199RQMDA3f9WSz0zMzM0KFDh2hoaEjqUoLKXLdRgSj4emHtdju0Wi1sNttd9wcBP45gV1VVhdLSUkRFRfEuNgPw46FXR0eHd6A7/ncxd3PdRvk8IgCVSoW8vDycOHGCx5NmXleuXMHY2Bj3+9xDHED4v/GDUlNTce7cOT41z2C329HS0oLCwkIe3+ce4gD6iSAIWL9+PSwWC4aHh6Uuh0nI4/GgsbERBQUF/DDBe4wD6AYRERF46KGH0NjY6HODLQsfRITOzk4sXryYHyboBxxAPxMVFeW9Poj7g8ILEWFwcBBXr17F2rVrOXz8gAPoZwRBgMFgQEJCArq6urg/KIw4HA60tLTwuM5+xAF0C7PjSV+5cgU//PADh1AY8Hg8aGhowKZNmxAdHS11OWGDA+gXREREoKioCK2trfx8sRBHRDh37hySk5ORkpIidTlhhQPoNrRaLXJzc9HY2MjjSYcoIsLQ0BCuXLmCdevWcb+Pn3EA/YrU1FQkJCTgzJkzfCgWghwOB9ra2rBt2zYe30cCvMZ/hSAIWLduHcxmM4aGhjiEQojb7capU6eQl5eH6Oho3vuRAAfQHERGRmLbtm3cHxRC6KfneSUkJGDx4sVSlxO2OIDmaHb8oLq6Oh4/KAQMDg5iYmICGzZs4D0fCXEAzdHs88WSkpL4efNBjIgwPj6OM2fOoKioiK/3kRgH0DzMDstgt9v5efNByuPx4NixYygoKODnuAcADqB5mn3efGdnJ8bGxnhPKIiIoojW1lZkZWVBr9fzoVcA4AC6A0qlEoWFhaivr+f+oCBBROjr64Pb7UZGRobU5bCfcADdAUEQkJCQgNWrV8NkMvFNqwGOiDAyMoL29nZ+jnuA4b/EHRIEAcuWLUNERAS6u7v5UCyAOZ1ONDc34ze/+Q0UCoXU5bAbcADdBUEQkJeXh2+++YYHMQtQHo8Hp06dwsaNGxEbGyt1OexnOIDuklwuxyOPPIK2tjZ+yGGAmR1UPi4uDqmpqdzpHIA4gBZAVFQUjEYj6uvr4XK5pC6H4cfw+f777+FwOPiJFgGMA2gBCIKApKQkLF26FM3NzdwpLTEiwvXr13Hx4kVs2rSJLzYMYBxAC0QQBGRmZkKpVHKntMQcDge+/PJLFBUVITIyUupy2G1wAC0gQRCQk5ODwcFB9Pf3cwhJwO12o6GhAcXFxYiPj+dDrwDHAbTAZDIZNm/ejLNnz/KV0n4miiKampqwdOlS6PV6qcthc8ABdA9ER0ejpKQEjY2NmJ6elrqcsEBE6OrqQkREBFauXMl7PkFiXgG0f/9+rF27FhqNBhqNBkajEcePH/cun56eRnl5ORISEhATE4Mnn3wSFovF5zMGBgawY8cOREVFITk5Ga+//npI3s4QGxuL7OxsVFVVYWZmRupyQhoRob+/H6OjoygoKODwCSLzCqDU1FS8/fbb6OjoQHt7O4qLi/H444+ju7sbAPDKK6/giy++wKFDh1BfX4+hoSHs2rXL+36Px4MdO3ZgZmYGTU1N+Pjjj/HRRx9h3759C/urAsDs455XrVqFpqYmPjN2jxARzGYzuru7+THKwYjuUnx8PP373/8mq9VKcrmcDh065F3W09NDAMhkMhER0bFjx0gmk5HZbPa22b9/P2k0GnI6nXP+TpvNRgDIZrPdbfn3nCiKdObMGWpvbydRFKUuJ+TY7Xb67LPPaGJiQupS2A3muo3ecR+Qx+PBwYMHMTk5CaPRiI6ODrhcLpSUlHjbZGRkwGAwwGQyAQBMJhOysrKg0+m8bcrKymC32717UaFmdkzp0dFRnD9/njulF9DU1BROnTqFBx98kJ/lFaTmHUBdXV2IiYmBUqnE7t27ceTIEWRmZsJsNkOhUCAuLs6nvU6ng9lsBgCYzWaf8JldPrvslzidTtjtdp8pmMhkMmzduhWjo6N8en6BOJ1OVFVVYd26dTf9m2PBY94BtHLlSnR2dqKlpQUvvvginnnmGVy8ePFe1OZVWVkJrVbrndLS0u7p990LkZGRKCoqQk9PD8xmM4fQXfB4PGhsbMTGjRuxZMkS7nQOYvMOIIVCgWXLliE7OxuVlZVYt24d/vnPf0Kv12NmZgZWq9WnvcVi8V6TodfrbzorNvv6dtdtVFRUwGazeafBwcH5lh0Q5HI5tmzZgvb2dly/fl3qcoLSbPikpqbivvvuk7ocdpfu+jogURThdDqRnZ0NuVyOmpoa77Le3l4MDAzAaDQCAIxGI7q6ujAyMuJtU11dDY1Gg8zMzF/8DqVS6T31PzsFq6ioKGzduhVtbW0YGxuTupygIooi2tvboVarsWLFCt7zCQXz6dl+4403qL6+ni5dukTnz5+nN954gwRBoK+++oqIiHbv3k0Gg4Fqa2upvb2djEYjGY1G7/vdbjetWbOGSktLqbOzk6qqqigpKYkqKiruSQ97oBJFkaxWKx08eJBGR0f57NgceDweam9vp87OTvJ4PFKXw37FXLfReQXQc889R/fddx8pFApKSkqibdu2ecOHiMjhcNBLL71E8fHxFBUVRTt37qTh4WGfz+jv76ft27eTWq2mxMREeu2118jlcs2njKAPoFnXr1+nqqoqstvtUpcS0DweDzU1NdHp06c5fILEXLdRgSj4ekPtdju0Wi1sNltQH44BwNjYGE6dOoWioiK+efIWRFFEV1cXZmZmkJ2dzeM5B4m5bqP815RYXFwcjEYjmpqaMD4+LnU5AUUURTQ3N8PlcnH4hCj+i0psdjCzwsJCNDU1YXR0lE/R48fwOXPmDABg48aNHD4hiv+qASIuLg75+fk4ceJE2J+i93g8aG5uhlwu58fohDj+ywYIQRAQHx+P7du3o62tDcPDw2G5J+RyuVBfXw+lUom1a9dy+IQ4/usGmJiYGGzduhVnzpxBX19fWIXQ5OQkamtrsXTpUh5IPkxwAAUYQRCgUqlQWlqK4eFhnD9/PuSH8iAi2Gw2HD9+HKtWrUJ6ejqHT5jgAApQcrkchYWFmJqaQl1dXcg+7oeIcPnyZdTX16O4uBgGg4HDJ4xwAAWwiIgIFBQUwGAwoLq6Gna7PaQOyURRxIULF9DR0YHS0lLEx8dLXRLzMw6gACcIAh544AHk5OSgqqoKg4ODQR9CRITp6WnU1tbC4XDg8ccfh0ql4j2fMMQBFARmrxV67LHH0Nvbi8bGxqAdR5uIMDIygqNHjyIlJQW5ubn87K4wxgEUJGY7p7dt2waVSoXPP/8c165dC6q9IZfLhVOnTqGlpQVlZWVYtWoV7/WEOf5fT5CRyWTIzc3FsmXLUFtbi+TkZBQUFCAyMjJgN2YiwvDwMFpbW5GamsqDxzMvvhk1SBERPB4Pzp8/j0uXLiEnJwdpaWkBd+He1NQU2traMDQ0hO3bt0Or1QZsULKFM9dtlAMoyBERxsfH0djYCFEUUVBQgISEBEk3ciKC2+3GuXPn8M033yAnJwcPPPAA7/WEEQ6gMDPbuVtbW4vY2Fjk5+cjMTHR70HkdDrR1dWFvr4+PPDAA8jKyoJSqeS9njDDARSmPB6Pt7/F7XYjLy8PS5Ysuad9RLN7YV1dXRgcHERqaio2bNiAqKgoDp4wxQEUxujHkS5x9epV797IqlWrsGLFCiQnJ0Mmk91VMMz+k3E4HOjr68PFixchk8mwfPlyrF69GnK5nIMnzHEAMQA/hoXT6cTAwAA6OzvhcDiQmJiIlJQU3HfffVCr1d5DpF8Kjdk+HafTidHRUVy+fBl9fX1QKBRIS0tDRkYGEhMTA64DnEmHA4jdhIgwMzODkZER9PX1YWxsDGazGTExMRAEAbGxsVCpVHC73VCpVLBarZDJZJicnIQoilCpVIiOjsby5cuRlJSEuLg4Dh12S3PdRvk6oDAiCAKUSiXS0tKQlpYGIoIoihBFEePj43A4HHC5XHA6nVAoFCAiaDQaxMTEeA+rbrenxNh8cQCFMUEQEBERgYiICCxatEjqclgY4v1nxphkOIAYY5LhAGKMSYYDiDEmGQ4gxphkOIAYY5LhAGKMSYYDiDEmGQ4gxphkOIAYY5IJylsxZu+ftdvtElfCGLuV2W3z1+51D8oAun79OgAgLS1N4koYY7czPj4OrVb7i8uDMoBmb5wcGBi47Y9jvux2O9LS0jA4OMjDmMwRr7M7MztKZkpKym3bBWUAzY5Bo9Vq+R/FHdBoNLze5onX2fzNZeeAO6EZY5LhAGKMSSYoA0ipVOKtt96CUqmUupSgwutt/nid3VtBOSY0Yyw0BOUeEGMsNHAAMcYkwwHEGJMMBxBjTDJBGUDvv/8+li5dCpVKhfz8fLS2tkpdkmQqKyuRm5uL2NhYJCcn44knnkBvb69Pm+npaZSXlyMhIQExMTF48sknYbFYfNoMDAxgx44diIqKQnJyMl5//XW43W5//hTJvP322xAEAXv37vXO43XmJxRkDh48SAqFgj788EPq7u6m559/nuLi4shisUhdmiTKysrowIEDdOHCBers7KRHH32UDAYDTUxMeNvs3r2b0tLSqKamhtrb26mgoIA2bdrkXe52u2nNmjVUUlJCZ8+epWPHjlFiYiJVVFRI8ZP8qrW1lZYuXUpr166ll19+2Tuf15l/BF0A5eXlUXl5ufe1x+OhlJQUqqyslLCqwDEyMkIAqL6+noiIrFYryeVyOnTokLdNT08PASCTyURERMeOHSOZTEZms9nbZv/+/aTRaMjpdPr3B/jR+Pg4LV++nKqrq2nz5s3eAOJ15j9BdQg2MzODjo4OlJSUeOfJZDKUlJTAZDJJWFngsNlsAP7vht2Ojg64XC6fdZaRkQGDweBdZyaTCVlZWdDpdN42ZWVlsNvt6O7u9mP1/lVeXo4dO3b4rBuA15k/BdXNqNeuXYPH4/H5owOATqfD119/LVFVgUMURezduxeFhYVYs2YNAMBsNkOhUCAuLs6nrU6ng9ls9ra51TqdXRaKDh48iDNnzqCtre2mZbzO/CeoAojdXnl5OS5cuIDGxkapSwlog4ODePnll1FdXQ2VSiV1OWEtqA7BEhMTERERcdPZCIvFAr1eL1FVgWHPnj04evQoTp48idTUVO98vV6PmZkZWK1Wn/Y3rjO9Xn/LdTq7LNR0dHRgZGQEGzduRGRkJCIjI1FfX493330XkZGR0Ol0vM78JKgCSKFQIDs7GzU1Nd55oiiipqYGRqNRwsqkQ0TYs2cPjhw5gtraWqSnp/ssz87Ohlwu91lnvb29GBgY8K4zo9GIrq4ujIyMeNtUV1dDo9EgMzPTPz/Ej7Zt24auri50dnZ6p5ycHDz99NPe/+Z15idS94LP18GDB0mpVNJHH31EFy9epBdeeIHi4uJ8zkaEkxdffJG0Wi3V1dXR8PCwd5qamvK22b17NxkMBqqtraX29nYyGo1kNBq9y2dPKZeWllJnZydVVVVRUlJSWJ1SvvEsGBGvM38JugAiIvrXv/5FBoOBFAoF5eXlUXNzs9QlSQbALacDBw542zgcDnrppZcoPj6eoqKiaOfOnTQ8POzzOf39/bR9+3ZSq9WUmJhIr732GrlcLj//Gun8PIB4nfkHD8fBGJNMUPUBMcZCCwcQY0wyHECMMclwADHGJMMBxBiTDAcQY0wyHECMMclwADHGJMMBxBiTDAcQY0wyHECMMclwADHGJPP/AbBB+oujwXChAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34.0 67 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
